package com.jackingod.crawler;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class RegexCrawler {

	private String url;
	public String htmlContent;
	public String NewsTitle;
	public String NewsContent;

	private String NewLine = System.getProperty("line.separator");

	public void start(String url) {
		init();
		this.url = url;
		this.startCrawling();
		this.extractNews();
	}
	
	private void init() {
		this.htmlContent = null;
		this.NewsTitle = null;
		this.NewsContent = null;
	}

	private void startCrawling() {
		StringBuffer sb = new StringBuffer();
		InputStreamReader isr = null;
		BufferedReader br = null;

		try {
			URL newsUrl = new URL(url);

			isr = new InputStreamReader(newsUrl.openStream());
			while (!isr.ready()) {
				// wait until the full web page has been downloaded.
			}

			br = new BufferedReader(isr);
			while (br.ready()) {
				sb.append(br.readLine()).append(NewLine);
			}

			this.htmlContent = sb.toString();

		} catch (MalformedURLException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
			try {
				if (br != null)
					br.close();
				if (isr != null)
					isr.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
	}
	
	private String removeTag(String html) {
		return html.replaceAll("\\&[a-zA-Z]{1,10};", "")
				.replaceAll("<[^>]*>", "").replaceAll("[(/>)<]", "");
	}

	private void extractNews() {
		if (this.htmlContent == null)
			return;

		String regex = ".*<!--\\sbegin_t\\s-->(.*)<!--\\send_t\\s-->" 
				+ ".*<!--\\sbegin_ct\\s-->(.*)<!--\\send_ct\\s-->.*";
		
		Pattern p = Pattern.compile(regex);
		Matcher m = p.matcher(htmlContent.replaceAll(NewLine, ""));
		if (m.matches()) {
			NewsTitle = m.group(1);
			NewsContent = m.group(2);
			NewsContent = removeTag(NewsContent);
		}
	}
}
